# -*-coding:utf-8-*-
import urllib.request
from bs4 import BeautifulSoup
import sys
#下面一句为转码语句，防止出现无法输出的特殊字符
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
def getHtml(url):
    """获取url页面"""
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}
    req = urllib.request.Request(url,headers=headers)
    req = urllib.request.urlopen(req)
    content = req.read().decode('utf-8')

    return content


def getComment(url):
    """解析HTML页面"""
    html = getHtml(url)
    soupComment = BeautifulSoup(html, 'html.parser')#创建bs对象

    comments = soupComment.findAll('span', 'short')
    
    
    onePageComments = []#将影评提取出来放在此列表中
    for comment in comments:#提取评论
        # print(comment.getText()+'\n')
        onePageComments.append(comment.getText()+'\n')

    return onePageComments


if __name__ == '__main__': 
    for page in range(10):  # 豆瓣爬取多页评论需要验证。
        url = 'https://movie.douban.com/subject/27119724/comments?start=' + str(20*page) + '&limit=20&sort=new_score&status=P'
        print('第%s页的评论:' % (page+1))
        print(url + '\n')

        for i in getComment(url):
            i=str(i).translate(non_bmp_map)#输出前转码
            print(i)#i即为评论内容
        print('\n')


